import os
import re
import time
from datetime import datetime

import logzero
import requests
from logzero import logger
from tqdm import tqdm
from lxml import html

from config import REPORT_ERROR_COLLECTION_NAME, COLLECTION_FEED
from utils import get_mongo_collection, record_run_status

pwd = os.path.dirname(os.path.abspath(__file__))
current_module = os.path.splitext(os.path.basename(__file__))[0]
COLLECTION_NAME = current_module
CSV_NAME = os.path.join(pwd, f'{current_module}.csv')
logzero.logfile(f"/tmp/{current_module}.log", maxBytes=30_000_000, backupCount=3, encoding='utf-8')

db = get_mongo_collection(COLLECTION_NAME)
error_db = get_mongo_collection(REPORT_ERROR_COLLECTION_NAME)
RE_permalink = re.compile('permalink="(.+?)";')
insert_count = 0


def spider(feeds):
    # 乐乎分析： http://note.youdao.com/noteshare?id=419faa8f13c0ca117cf22013b3a1d931
    for user_name, name, uid in feeds:
        all_posts = get_all_posts(user_name, uid)
        for posts in tqdm(all_posts):
            data = get_detail(posts, user_name, name, uid)
            stop = store_data(data)
            if stop: break
    logger.info('完成')


def get_feeds():
    db_feed = get_mongo_collection(COLLECTION_FEED)
    cursor = db_feed.find({'purpose': '乐乎'})
    feeds = []
    for cu in cursor:
        feeds.append((cu['user_name'], cu['name'], cu['uid']))
    return feeds


def get_all_posts(user_name, uid):
    url = f'http://{user_name}.lofter.com/dwr/call/plaincall/ArchiveBean.getArchivePostByTime.dwr'
    Referer = f'http://{user_name}.lofter.com/view'

    ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 ' \
         'Safari/537.36'
    headers = {'Referer': Referer, 'User-Agent': ua}
    t = int(time.time() * 1000)
    data = f"""
callCount=1
scriptSessionId=${{scriptSessionId}}187
httpSessionId=
c0-scriptName=ArchiveBean
c0-methodName=getArchivePostByTime
c0-id=0
c0-param0=number:{uid}
c0-param1=number:{t}
c0-param2=number:1000
c0-param3=boolean:false
batchId=210338
"""
    logger.info(f'请求 {url}')
    resp = requests.post(url, data=data, headers=headers).text
    all_posts = RE_permalink.findall(resp)
    all_posts = [f'http://{user_name}.lofter.com/post/' + h for h in all_posts]
    if len(all_posts) == 0:
        logger.info(f'url {url}')
        logger.info(data)
    return all_posts


def get_detail(posts, user_name, name, uid):
    logger.info(f'请求 {posts}')
    resp = requests.get(posts).text
    root = html.fromstring(resp)
    pics = []
    desp = ''
    themes = [
        ('div.content div.img>a>img', 'div.content div.text>p'),
        ('div.box>div.pic>a>img', 'div.box>div.text>p'),
        ('div.box>div.pic>div.img>a>img', 'div.box>div.text>p'),
        ('div.cont>div.pic>a>img', 'div.cont>div.text>p'),
        ('div.cont>div.pic>div.img>a>img', 'div.cont>div.text>p'),
        ('div.post-ctc div.pic>a>img', 'div.post-ctc div.text>p'),
        ('div.postphoto div.PhotoWrapper>a>img', 'div.postphoto div.photopostdesc>p'),
    ]
    for img_css, p_css in themes:
        imgs = root.cssselect(img_css)
        for img in imgs:
            src = img.attrib['src']
            pics.append(src)
        desp_ele = root.cssselect(p_css)
        for p in desp_ele:
            if desp:
                desp += p.text + '\n'
        if len(pics) > 0:
            break
    data = {'user_name': user_name, 'name': name, 'uid': uid, 'url': posts, 'pics': pics, 'desp': desp,
            'update_time': datetime.now()}
    return data


def store_data(data):
    global insert_count
    stop = False
    url = data['url']
    if len(data['pics']) == 0:
        logger.info('没有图片，跳过')
        return stop
    old_record = db.find_one({'url': url})
    if old_record:
        logger.info(f'{url} 已经存在，不再爬取')
        stop = True
    else:
        logger.info(f'{url} 不存在，现在插入')
        logger.info(data)
        db.insert_one(data)
        insert_count += 1
    return stop


@record_run_status(current_module)
def main():
    feeds = get_feeds()
    spider(feeds)
    return insert_count


if __name__ == '__main__':
    main()
